{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a2a2297d-ad90-4a4b-a0c9-01cb3028469c",
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from openai import OpenAI\n",
    "from sklearn.metrics import matthews_corrcoef\n",
    "from tqdm import tqdm\n",
    "import torch\n",
    "from transformers import pipeline, AutoTokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "e6a7c9b0",
   "metadata": {},
   "outputs": [],
   "source": [
    "trump = pd.read_csv('./trump_test_in_context.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "45d95ed0-b020-4803-aa9c-84a3aba3cc8d",
   "metadata": {},
   "source": [
    "# GPT-4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "375e3e0f-652b-4d4a-afb8-803dc537cfb6",
   "metadata": {},
   "outputs": [],
   "source": [
    "client = OpenAI(api_key = '')\n",
    "system_message =  \"\"\"You are a text classifier and are only allowed to respond with a 1 or a 0.\"\"\"\n",
    "user_message = \"\"\"You are a classifier that determines if the author of a text supports Donald Trump. I will post text about Trump and if the author is more likely to support Trump than not return 1. Otherwise, return 0. Do not explain the classification or say anything else. You are only allowed to respond with a 1 or a 0. Here is the text. \\nText: {}\"\"\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ce527425",
   "metadata": {},
   "source": [
    "No Logit Bias"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cfac4341",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "scrolled": true
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'client' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)\n",
      "File \u001b[1;32m<timed exec>:4\u001b[0m\n",
      "\n",
      "\u001b[1;31mNameError\u001b[0m: name 'client' is not defined"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "labels = []\n",
    "for text in trump['text']:\n",
    "    # query API\n",
    "    res = client.chat.completions.create(\n",
    "        model = 'gpt-4', \n",
    "        messages = [\n",
    "            {'role':'system', 'content': system_message},\n",
    "            {'role':'user', 'content': user_message.format(text)}\n",
    "        ],\n",
    "        max_tokens = 1,\n",
    "        temperature = 0\n",
    "    )\n",
    "    labels.append(res.choices[0].message.content)\n",
    "\n",
    "labels = [int(label) for label in labels]\n",
    "trump['gpt4_nobias'] = labels\n",
    "trump.to_csv('./trump_test_in_context.csv', index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "620214f5-7f05-47ee-8d7c-d10de0dc5a5c",
   "metadata": {},
   "source": [
    "Logit Bias"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d97aa507-e0ca-4823-ab41-48c0cf663642",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "labels = []\n",
    "for text in trump['text']:\n",
    "    # query API\n",
    "    res = client.chat.completions.create(\n",
    "        model = 'gpt-4', \n",
    "        messages = [\n",
    "            {'role':'system', 'content': system_message},\n",
    "            {'role':'user', 'content': user_message.format(text)}\n",
    "        ],\n",
    "        max_tokens = 1,\n",
    "        temperature = 0,\n",
    "        logit_bias = {16:100, 15:100}\n",
    "    )\n",
    "    labels.append(res.choices[0].message.content)\n",
    "\n",
    "labels = [int(label) for label in labels]\n",
    "trump['gpt4_bias'] = labels\n",
    "trump.to_csv('./trump_test_in_context.csv', index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "137f3b4d-2564-4fd3-9fc3-09519f505414",
   "metadata": {},
   "source": [
    "Chain of Thought"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "50f2129f-274f-4dc5-b178-22ff9f121f45",
   "metadata": {},
   "outputs": [],
   "source": [
    "system_message = \"\"\"\n",
    "You are a text classifier that determines the political beliefs of a document's author. Follow these steps to answer the user queries:\n",
    "\n",
    "Step 1 - First explain how the language in the text sample indicates what the author believes about the topic. Enclose all of your work in this step with double quotes(\"\").\n",
    "\n",
    "Step 2 - After providing an explanation, write the most appropriate label on a new line. This new line should only contain the chosen label. After you have written the label on a new line, end your answer and do not write anything else.\n",
    "\"\"\"\n",
    "\n",
    "user_message = \"\"\"Directions: Given the text sample about Donald Trump: \"{}\", first, consider what the author's opinion about Trump is. Then classify the author's stance toward Trump into one of the following categories: \"Support\", \"Oppose\", \"Neutral\". The definitions of the categories are\n",
    "\"Support\": The author uses language that indicates they support or approve of Trump. This may include explicit statements of support for him or his policies, or any language that might indicate a positive view of Trump.\n",
    "\"Oppose\": The author uses language that indicates they oppose or disapprove of Trump. This may include explicit criticism for him or his policies, or any language that might indicate that the author disapproves of Trump.\n",
    "\"Neutral\": There is no language the might indicate positive or negative sentiment towards Trump from the author.\n",
    "Briefly explain the stance of the author towards Trump with a response that does not exceed 100 words.\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "86f7a422-f92d-4c41-97f8-ec77fb4bf73a",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "# 6hrs 18min for COT, $38.62\n",
    "labels = []\n",
    "for text in trump['text']:\n",
    "    # query API\n",
    "    res = client.chat.completions.create(\n",
    "        model = 'gpt-4', \n",
    "        messages = [\n",
    "            {'role':'system', 'content': system_message},\n",
    "            {'role':'user', 'content': user_message.format(text)}\n",
    "        ],\n",
    "        max_tokens = 200,\n",
    "        temperature = 0\n",
    "        #logit_bias = {16:100, 15:100}\n",
    "    )\n",
    "    labels.append(res.choices[0].message.content)\n",
    "\n",
    "trump['gpt4_cot'] = [1 if 'support' in label.lower()[-10:] else 0 for label in labels]\n",
    "trump.to_csv('./trump_test_in_context.csv', index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0ac928c9-f832-4683-bc14-46cbc68c832c",
   "metadata": {},
   "source": [
    "# GPT 3.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ced42e08",
   "metadata": {},
   "outputs": [],
   "source": [
    "client = OpenAI(api_key = '')\n",
    "system_message =  \"\"\"You are a text classifier and are only allowed to respond with a 1 or a 0.\"\"\"\n",
    "user_message = \"\"\"You are a classifier that determines if the author of a text supports Donald Trump. I will post text about Trump and if the author is more likely to support Trump than not return 1. Otherwise, return 0. Do not explain the classification or say anything else. You are only allowed to respond with a 1 or a 0. Here is the text. \\nText: {}\"\"\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "afba1eea-aa11-48a0-942b-e28f9deea12d",
   "metadata": {},
   "source": [
    "No Logit Bias"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b3c2d67d-8ccd-487c-87fc-d6bbfa81fa23",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "labels = []\n",
    "for text in trump['text']:\n",
    "    # query API\n",
    "    res = client.chat.completions.create(\n",
    "        model = 'gpt-3.5-turbo', \n",
    "        messages = [\n",
    "            {'role':'system', 'content': system_message},\n",
    "            {'role':'user', 'content':user_message.format(text)}\n",
    "        ],\n",
    "        max_tokens = 1,\n",
    "        temperature = 0\n",
    "    )\n",
    "    labels.append(res.choices[0].message.content)\n",
    "\n",
    "labels = [int(label) for label in labels]\n",
    "trump['gpt3_5_nobias'] = labels\n",
    "trump.to_csv('./trump_test_in_context.csv', index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1ecd6ba4-14a5-4c4d-8338-9ff1f3c2ba41",
   "metadata": {},
   "source": [
    "Logit Bias"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "id": "4d31bbe8-c19d-469a-b3b8-07c05f7d0425",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 8.23 s\n",
      "Wall time: 6h 49min 34s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "labels = []\n",
    "for text in trump['text']:\n",
    "    # query API\n",
    "    res = client.chat.completions.create(\n",
    "        model = 'gpt-3.5-turbo', \n",
    "        messages = [\n",
    "            {'role':'system', 'content': system_message},\n",
    "            {'role':'user', 'content':user_message.format(text)}\n",
    "        ],\n",
    "        max_tokens = 1,\n",
    "        temperature = 0,\n",
    "        logit_bias = {16:100, 15:100}\n",
    "    )\n",
    "    labels.append(res.choices[0].message.content)\n",
    "labels = [int(label) for label in labels]\n",
    "trump['gpt3_5_bias'] = labels\n",
    "trump.to_csv('./trump_test_in_context.csv', index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dc660ca1-937a-4f9d-8405-186961ca038a",
   "metadata": {},
   "source": [
    "Chain of Thought"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "c2381b30-3427-445d-bea4-7b23e3dcd820",
   "metadata": {},
   "outputs": [],
   "source": [
    "system = \"\"\"\n",
    "You are a text classifier that determines the political beliefs of a document's author. Follow these steps to answer the user queries:\n",
    "\n",
    "Step 1 - First explain how the language in the text sample indicates what the author believes about the topic. Enclose all of your work in this step with double quotes(\"\").\n",
    "\n",
    "Step 2 - After providing an explanation, write the most appropriate label on a new line. This new line should only contain the chosen label. After you have written the label on a new line, end your answer and do not write anything else.\n",
    "\"\"\"\n",
    "\n",
    "prompt = \"\"\"Directions: Given the text sample about Donald Trump: \"{}\", first, consider what the author's opinion about Trump is. Then classify the author's stance toward Trump into one of the following categories: \"Support\", \"Oppose\", \"Neutral\". The definitions of the categories are\n",
    "\"Support\": The author uses language that indicates they support or approve of Trump. This may include explicit statements of support for him or his policies, or any language that might indicate a positive view of Trump.\n",
    "\"Oppose\": The author uses language that indicates they oppose or disapprove of Trump. This may include explicit criticism for him or his policies, or any language that might indicate that the author disapproves of Trump.\n",
    "\"Neutral\": There is no language the might indicate positive or negative sentiment towards Trump from the author.\n",
    "Briefly explain the stance of the author towards Trump with a response that does not exceed 100 words.\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "263dd962-7a70-4cda-a105-c48d246a2115",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 8.64 s\n",
      "Wall time: 4h 20min 27s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# 28 responses did not conform to the COT prompt template\n",
    "labels = []\n",
    "for text in trump['text']:\n",
    "    # query API\n",
    "    res = client.chat.completions.create(\n",
    "        model = 'gpt-3.5-turbo', \n",
    "        messages = [\n",
    "            {'role':'system', 'content': system},\n",
    "            {'role':'user', 'content': prompt.format(text)}\n",
    "        ],\n",
    "        max_tokens = 200,\n",
    "        temperature = 0\n",
    "        #logit_bias = {16:100, 15:100}\n",
    "    )\n",
    "    labels.append(res.choices[0].message.content)\n",
    "trump['gpt3_5_cot'] = [1 if 'support' in label.lower()[-10:] else 0 for label in labels]\n",
    "trump.to_csv('./trump_test_in_context.csv', index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dadd84f3-2293-426c-8f15-364962a1e9eb",
   "metadata": {},
   "source": [
    "# Mistral 7B"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7e4271dc-de75-4376-bad8-a0101ce39b95",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\mikeb\\.conda\\envs\\sandbox\\Lib\\site-packages\\huggingface_hub\\file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fd66f793923845b6be41fda618826763",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/8 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "model = \"HuggingFaceH4/zephyr-7b-beta\"\n",
    "pipe = pipeline(\"text-generation\", model=model, torch_dtype=torch.bfloat16, device_map=\"auto\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "380f9b5d-6089-40cd-b83a-1eaaad38f0a4",
   "metadata": {},
   "source": [
    "No Logit Bias"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "4baa71cc-5910-4303-a21d-b9161551cb57",
   "metadata": {},
   "outputs": [],
   "source": [
    "system_message =  \"\"\"You are a text classifier and are only allowed to respond with a 1 or a 0.\"\"\"\n",
    "user_message = \"\"\"You are a classifier that determines if the author of a text supports Donald Trump. I will post text about Trump and if the author is more likely to support Trump than not return 1. Otherwise, return 0. Do not explain the classification or say anything else. You are only allowed to respond with a 1 or a 0. Here is the text. \\nText: {}\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "513b7a34-108f-423d-a2b4-05f8c8764fa2",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\mikeb\\.conda\\envs\\sandbox\\Lib\\site-packages\\transformers\\generation\\utils.py:1473: UserWarning: You have modified the pretrained model configuration to control generation. This is a deprecated strategy to control generation and will be removed soon, in a future version. Please use and modify the model generation configuration (see https://huggingface.co/docs/transformers/generation_strategies#default-text-generation-configuration )\n",
      "  warnings.warn(\n",
      "c:\\Users\\mikeb\\.conda\\envs\\sandbox\\Lib\\site-packages\\transformers\\generation\\configuration_utils.py:381: UserWarning: `do_sample` is set to `False`. However, `temperature` is set to `0` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `temperature`.\n",
      "  warnings.warn(\n",
      "c:\\Users\\mikeb\\.conda\\envs\\sandbox\\Lib\\site-packages\\transformers\\pipelines\\base.py:1101: UserWarning: You seem to be using the pipelines sequentially on GPU. In order to maximize efficiency please use a dataset\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'0', 'Return', '1'}\n",
      "1\n",
      "CPU times: total: 2min 22s\n",
      "Wall time: 2min 32s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "res = []\n",
    "for doc in trump['text']:\n",
    "    messages = [\n",
    "        {\n",
    "            \"role\": \"system\",\n",
    "            \"content\": system_message,\n",
    "        },\n",
    "        {\"role\": \"user\", \"content\": user_message.format(doc)},\n",
    "    ]\n",
    "    prompt = pipe.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)\n",
    "    outputs = pipe(prompt, max_new_tokens=1, do_sample=False, return_full_text = False, pad_token_id=pipe.tokenizer.eos_token_id, temperature = 0)\n",
    "    res.extend(outputs)\n",
    "\n",
    "res = [text['generated_text'] for text in res]\n",
    "\n",
    "# return a list of unique responses from the model\n",
    "print(set(res))\n",
    "# count non-compliant responses\n",
    "print(sum([1 if 'Return' in text else 0 for text in res]))\n",
    "# add results to the dataframe\n",
    "trump['mistral_nobias'] = [1 if '1' in text else 0 for text in res]\n",
    "# export results\n",
    "trump.to_csv('./trump_test_in_context.csv', index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3d45d813-2af1-48b8-be5e-63e0c0e60dee",
   "metadata": {},
   "source": [
    "Logit Bias"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "dd6b13ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "# define tokenizer\n",
    "tokenizer_with_prefix_space = AutoTokenizer.from_pretrained(model, add_prefix_space=True)\n",
    "# helper function for tokenizing the words I want to bias\n",
    "def get_tokens_as_tuple(word):\n",
    "    return tuple(tokenizer_with_prefix_space([word], add_special_tokens=False).input_ids[0])\n",
    "# get a dictionary of biased tokens with their associated bias\n",
    "sequence_bias = {get_tokens_as_tuple(\"1\"): 10.0, get_tokens_as_tuple(\"0\"): 10.0}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "8910a7d3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'0', 'Return', '1'}\n",
      "1\n",
      "CPU times: total: 2min 22s\n",
      "Wall time: 2min 33s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "res = []\n",
    "for doc in trump['text']:\n",
    "    messages = [\n",
    "        {\n",
    "            \"role\": \"system\",\n",
    "            \"content\": system_message,\n",
    "        },\n",
    "        {\"role\": \"user\", \"content\": user_message.format(doc)},\n",
    "    ]\n",
    "    prompt = pipe.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)\n",
    "    outputs = pipe(prompt, max_new_tokens=1, do_sample=False, return_full_text = False, sequence_bias=sequence_bias, pad_token_id=pipe.tokenizer.eos_token_id, temperature = 0)\n",
    "    res.extend(outputs)\n",
    "# check for non-compliance by checking if all responses are 1 or 0\n",
    "res = [text['generated_text'] for text in res]\n",
    "print(set(res))\n",
    "print(sum([1 if 'Return' in text else 0 for text in res]))\n",
    "# add results to the dataframe\n",
    "trump['mistral_bias'] = [1 if '1' in text else 0 for text in res]\n",
    "trump.to_csv('./trump_test_in_context.csv', index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2bb13ca8-5fa8-4b1e-89a7-7fd3684744dc",
   "metadata": {},
   "source": [
    "Chain of Thought"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "978833a7",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt = \"\"\"\n",
    "You are a text classifier that determines the political beliefs of a document's author. Follow these steps to answer the user queries:\n",
    "\n",
    "Step 1 - First explain how the language in the text sample indicates what the author believes about the topic. Enclose all of your work in this step with double quotes(\"\").\n",
    "\n",
    "Step 2 - After providing an explanation, write the most appropriate label on a new line. This new line should only contain the chosen label. After you have written the label on a new line, end your answer and do not write anything else.\n",
    "\n",
    "Directions: Given the text sample about Donald Trump: \"{}\", first, consider what the author's opinion about Trump is. Then classify the author's stance toward Trump into one of the following categories: \"Support\", \"Oppose\", \"Neutral\". The definitions of the categories are\n",
    "\"Support\": The author uses language that indicates they support or approve of Trump. This may include explicit statements of support for him or his policies, or any language that might indicate a positive view of Trump.\n",
    "\"Oppose\": The author uses language that indicates they oppose or disapprove of Trump. This may include explicit criticism for him or his policies, or any language that might indicate that the author disapproves of Trump.\n",
    "\"Neutral\": There is no language the might indicate positive or negative sentiment towards Trump from the author.\n",
    "Briefly explain the stance of the author towards Trump with a response that does not exceed 100 words.\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "cec944f7",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing documents: 100%|██████████| 2134/2134 [1:50:31<00:00,  3.11s/it]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 1h 42min 24s\n",
      "Wall time: 1h 50min 31s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "res = []\n",
    "for doc in tqdm(trump['text'], desc=\"Processing documents\"):\n",
    "    messages = [\n",
    "        {\"role\": \"user\", \"content\": prompt.format(doc)},\n",
    "    ]\n",
    "    prompt_prepared = pipe.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)\n",
    "    outputs = pipe(prompt_prepared, max_new_tokens=200, do_sample=False, return_full_text=False, pad_token_id=pipe.tokenizer.eos_token_id)\n",
    "    res.extend(outputs)\n",
    "\n",
    "trump['mistral_cot'] = [1 if 'support' in label['generated_text'].lower()[-20:] else 0 for label in res]\n",
    "trump.to_csv('./trump_test_in_context.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "ed4d13bd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.872539831302718"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check compliance\n",
    "compliance = [1 if any(word in label['generated_text'].lower()[-20:] for word in ['support', 'oppose', 'neutral']) else 0 for label in res]\n",
    "sum(compliance)/len(compliance)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
